# -*- coding: utf-8 -*-
import os
import re

import scrapy


class OsgeoSpider(scrapy.Spider):
    name = 'osgeo'
    allowed_domains = ['www.osgeo.cn']
    start_urls = ['https://www.osgeo.cn/scrapy/index.html']

    def parse(self, response):
        li_list = response.xpath("//ul/li[@class='toctree-l1']")
        # print(li_list)
        for li_temp in li_list:
            html_url = li_temp.xpath('./a/@href').extract_first()
            html_url_text = 'https://www.osgeo.cn/scrapy/' + html_url
            print(html_url_text)

            yield scrapy.Request(url=html_url_text, callback=self.get_css_js,
                                 cb_kwargs={"name": html_url_text})

    def get_css_js(self, response, name):
        # print(1111)
        print('html------->', response.url)
        download_path = os.getcwd() + '/html/'
        if not os.path.exists(download_path):
            os.makedirs(download_path)
        body_url = response.body.decode()
        index = name.rfind('/')
        name_str = name[index + 1:]
        with open('./html/' + name_str, "w", encoding='utf-8')as f:
            f.write(body_url)

        # print(body_url)
        ret_js = re.findall(r'.* <script src="(.*?)"></script>', str(body_url))
        for js_url in ret_js:
            js_url_str = js_url.replace('../', '')
            js_url_str_2 = "https://www.osgeo.cn/scrapy/" + js_url_str
            print('js------->', js_url_str_2)
            yield scrapy.Request(url=js_url_str_2, callback=self.write_js, cb_kwargs={"name": js_url})

        ret_css = re.findall(r'.* <link rel="stylesheet" href="(.*)" type="text/css" />', body_url)

        for css_url in ret_css:
            css_url_text = css_url.replace('../', '')
            css_url_str = "https://www.osgeo.cn/scrapy/" + css_url_text
            print('css------>', css_url_str)
            yield scrapy.Request(url=css_url_str, callback=self.write_css, cb_kwargs={"name": css_url})

    def write_js(self, response, name):
        # print(response.url)
        download_path = os.getcwd() + '/_static/js/'
        if not os.path.exists(download_path):
            os.makedirs(download_path)
        body = response.body.decode()
        index = name.rfind('/')
        name_str = name[index + 1:]
        with open('./_static/js/' + name_str, "w", encoding='utf-8')as f:
            f.write(body)

    def write_css(self, response, name):
        # print(response.url)
        download_path = os.getcwd() + '/_static/css/'
        if not os.path.exists(download_path):
            os.makedirs(download_path)
        body = response.body.decode()
        index = name.rfind('/')
        name_str = name[index + 1:]
        with open('./_static/css/' + name_str, "w", encoding='utf-8')as f:
            f.write(body)
